﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Web;

namespace CrawlerCore
{
    public class CrawlerHelper
    {
        public HtmlDocument getDocument(string adresss, string method, string getOrPostData, CookieCollection cookieCollection)
        {
            HtmlDocument d = new HtmlDocument();
            if (method.ToLower() == "get")
            {
                adresss += getOrPostData;
            }
            Uri uri = new Uri(adresss);
            HttpWebRequest request = WebRequest.Create(uri) as HttpWebRequest;
            request.Method = method;
            request.Timeout = 100000;
            request.AllowAutoRedirect = true;
            request.MaximumAutomaticRedirections = 200;
            request.UseDefaultCredentials = true;

            request.KeepAlive = true;
            
            request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0";
            var cookieTemp = new CookieContainer();
            if (cookieCollection != null)
            {
                cookieTemp.Add(cookieCollection);
            }
            request.CookieContainer = cookieTemp;
            if (method.ToLower() == "post")
            {
                request.ContentType = "application/x-www-form-urlencoded";
                request.ContentLength = getOrPostData.Length;
                StreamWriter requestStream = new StreamWriter(request.GetRequestStream());
                requestStream.Write(getOrPostData);
                requestStream.Close();
            }
            //get response
            using( WebResponse response = request.GetResponse())
            {
                using (StreamReader reader = new StreamReader(response.GetResponseStream()))
                {
                    String html = reader.ReadToEnd();
                    reader.Close();
                    d.LoadHtml(html);
                }
                response.Close();
            }
            return d;
        }

        public CookieCollection getCookieCollection(string adresss, string method, string getOrPostData)
        {
            HttpWebRequest request;
            var cookieJar = new CookieContainer();

            if (method.ToLower() == "get")
            {
                adresss += getOrPostData;
            }
            Uri uri = new Uri(adresss);
            request = WebRequest.Create(uri) as HttpWebRequest;
            request.CookieContainer = cookieJar;
            request.Method = method;
            request.AllowAutoRedirect = true;
            request.MaximumAutomaticRedirections = 200;
            request.Timeout = 100000;

            if (method.ToLower() == "post")
            {
                request.ContentType = "application/x-www-form-urlencoded";
                request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36";
                request.ContentLength = getOrPostData.Length;
                StreamWriter requestStream = new StreamWriter(request.GetRequestStream());
                requestStream.Write(getOrPostData);
                requestStream.Close();
            }
            //get respon
            WebResponse response = request.GetResponse();
            StreamReader reader = new StreamReader(response.GetResponseStream());
            String html = reader.ReadToEnd();
            var cookiesCollection = new CookieCollection();
            foreach (Cookie c in cookieJar.GetCookies(request.RequestUri))//cookieJar.GetCookies(request.RequestUri)
            {
                cookiesCollection.Add(new Cookie(c.Name, c.Value, c.Path, c.Domain));
            }
            response.Close();
            return cookiesCollection;
        }
    }
}